import pandas as pd
import numpy as np
from sklearn import metrics
from sklearn.model_selection import cross_val_score, KFold, StratifiedShuffleSplit, ShuffleSplit
# Pytorch
import torch
from torch.autograd import Variable
import torch.nn as nn
import torchvision.transforms as transforms
# Visualisation libraries
## Progress Bar
import progressbar
## Text
from colorama import Fore, Back, Style
from IPython.display import Image, display, Markdown, Latex, clear_output
## plotly
from plotly.offline import init_notebook_mode, iplot
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px
## seaborn
import seaborn as sns
sns.set_style("whitegrid")
sns.set_context("paper", rc={"font.size":12,"axes.titlesize":14,"axes.labelsize":12})
## matplotlib
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
from matplotlib.patches import Ellipse, Polygon
import matplotlib.gridspec as gridspec
import matplotlib.colors
from pylab import rcParams
from matplotlib.font_manager import FontProperties
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
plt.style.use('seaborn-whitegrid')
import matplotlib as mpl
mpl.rcParams['figure.figsize'] = (17, 6)
mpl.rcParams['axes.labelsize'] = 14
mpl.rcParams['xtick.labelsize'] = 12
mpl.rcParams['ytick.labelsize'] = 12
mpl.rcParams['text.color'] = 'k'
%matplotlib inline
import warnings
warnings.filterwarnings("ignore")
Sign languages are languages that can include simultaneously employing hand gestures, movement, the orientation of the fingers, arms or body, and facial expressions to convey a speaker's ideas [1]. In this article, we develop a stacking classifier model from various scikit-learn classifiers for the Sign Language Digits dataset from Kaggle. The main purpose of this article is to demonstrate implementation of stacking classifiers.
Path = 'Sign_Language_Digits'
# loading the dataset
X = np.load(Path + '/X.npy')
y = np.load(Path + '/Y.npy')
Investigating the data, we can see that y represent numbers 0 to 9 that encoded into dummy variables.
_ , Values = np.nonzero(y[:,:])
Values = pd.Series(Values)
Ind = [Values[Values == i].index.min() for i in Values.unique()]
fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(17, 6), subplot_kw={'xticks': [], 'yticks': []})
_ = fig.suptitle('A Sample of Dataset', fontweight='bold', fontsize = 18)
ax = ax.ravel()
font = FontProperties()
font.set_family('fantasy')
Ind = [Values[Values == i].index.min() for i in Values.unique()]
for i in range(len(ax)):
_ = ax[i].imshow(X[Ind[i],:,:], cmap = 'gray')
_ = ax[i].set_title('%i' % np.nonzero(y[Ind[i],:]), fontproperties=font, fontsize = 16)
_ = ax[i].set_aspect(1)
fig.tight_layout()
y = Values.copy()
del Values
def Dist_Plot(y, Plot_dict):
def ToSeries(x):
if not isinstance(x, pd.Series):
Out = pd.Series(x)
else:
Out = x.copy()
return Out
fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths= Plot_dict['column_widths'],
specs=[[{"type": "table"},{"type": "pie"}]])
# Right
fig.add_trace(go.Pie(labels= np.sort(y.unique()),
values= ToSeries(y).value_counts().values, pull=[0.1],
textfont=dict(size=Plot_dict['textfont']),
marker=dict(colors = Plot_dict['PieColors'],
line=dict(color='black', width=1))), row=1, col=2)
fig.update_traces(hole=.5)
fig.update_layout(legend=dict(orientation="v"), legend_title_text= Plot_dict['legend_title'])
# Left
## Table
Table = y.value_counts().to_frame('Count').reset_index(drop = False)
Table = Table.rename(columns = {'index':Plot_dict['legend_title']}).sort_values(by = [Plot_dict['legend_title']])
Table['Percentage'] = np.round(100*(Table['Count']/Table['Count'].sum()),2)
T = Table.copy()
T['Percentage'] = T['Percentage'].map(lambda x: '%% %.2f' % x)
Temp = []
for i in T.columns:
Temp.append(T.loc[:,i].values)
TableColors = Plot_dict['TableColors']
fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
fill_color= TableColors[0], align=['center','center'],
font=dict(color='white', size=12), height=25), columnwidth = [0.2, 0.4, 0.4],
cells=dict(values=Temp, line_color='darkslategray',
fill=dict(color= [TableColors[1], TableColors[1]]),
align=['center', 'center'], font_size=12, height=20)), 1, 1)
fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
if not Plot_dict['height'] == None:
fig.update_layout(height = Plot_dict['height'])
fig.show()
Plot_dict = dict(column_widths = [0.3, 0.7], PieColors = None, textfont = 12,
TableColors = ['Navy','White'], height = 450, legend_title = 'Digits')
Dist_Plot(y, Plot_dict)
StratifiedKFold is a variation of k-fold which returns stratified folds: each set contains approximately the same percentage of samples of each target class as the complete set.
Labels = np.sort(y.unique())
Test_Size = 0.3
sss = StratifiedShuffleSplit(n_splits=1, test_size=Test_Size, random_state=42)
_ = sss.get_n_splits(X, y)
for train_index, test_index in sss.split(X, y):
# X
if isinstance(X, pd.DataFrame):
X_train, X_test = X.loc[train_index], X.loc[test_index]
else:
X_train, X_test = X[train_index], X[test_index]
# y
if isinstance(y, pd.Series):
y_train, y_test = y[train_index], y[test_index]
else:
y_train, y_test = y[train_index], y[test_index]
del sss
# Flattening the X sets
def Flat(Inp): return Inp.reshape(Inp.shape[0],Inp.shape[1]*Inp.shape[2])
X_train, X_test = Flat(X_train), Flat(X_test)
def Train_Test_Dist(X_train, y_train, X_test, y_test, Plot_dict):
def ToSeries(x):
if not isinstance(x, pd.Series):
Out = pd.Series(x)
else:
Out = x.copy()
return Out
fig = make_subplots(rows=1, cols=3, horizontal_spacing = 0.02, column_widths= Plot_dict['column_widths'],
specs=[[{"type": "table"},{'type':'domain'}, {'type':'domain'}]])
# Right
C = 2
for y in [y_train, y_test]:
fig.add_trace(go.Pie(labels= np.sort(y.unique()),
values= ToSeries(y).value_counts().values, pull=[0.1],
textfont=dict(size=Plot_dict['textfont']),
marker=dict(colors = Plot_dict['PieColors'],
line=dict(color='black', width=1))), row=1, col=C)
fig.update_traces(hole=.5)
fig.update_layout(legend=dict(orientation="v"), legend_title_text= Plot_dict['legend_title'])
C+=1
# Left
# Table
Table = pd.DataFrame(data={'Set':['X_train','X_test','y_train','y_test'],
'Shape':[X_train.shape, X_test.shape, y_train.shape, y_test.shape]}).astype(str)
T = Table.copy()
Temp = []
for i in T.columns:
Temp.append(T.loc[:,i].values)
TableColors = Plot_dict['TableColors']
fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
fill_color= TableColors[0], align=['center','center'],
font=dict(color='white', size=12), height=25), columnwidth = [0.2, 0.4],
cells=dict(values=Temp, line_color='darkslategray',
fill=dict(color= [TableColors[1], TableColors[1]]),
align=['center', 'center'], font_size=12, height=20)), 1, 1)
fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
if not Plot_dict['height'] == None:
fig.update_layout(height = Plot_dict['height'])
fig.show()
Plot_dict = dict(column_widths = [0.3, 0.3, 0.3], PieColors = None, textfont = 12,
TableColors = ['Navy','White'], height = 450, legend_title = 'Digits')
Train_Test_Dist(X_train, y_train, X_test, y_test, Plot_dict)
A multi-layer perceptron (MLP) is a class of feedforward artificial neural network (ANN). The algorithm at each iteration uses the Cross-Entropy Loss to measure the loss, and then the gradient and the model update is calculated. At the end of this iterative process, we would reach a better level of agreement between test and predicted sets since the error would be lower from that of the first step.
def Header(Text, L = 100, C = 'Blue', T = 'White'):
BACK = {'Black': Back.BLACK, 'Red':Back.RED, 'Green':Back.GREEN, 'Yellow': Back.YELLOW, 'Blue': Back.BLUE,
'Magenta':Back.MAGENTA, 'Cyan': Back.CYAN}
FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
print(BACK[C] + FORE[T] + Style.NORMAL + Text + Style.RESET_ALL + ' ' + FORE[C] +
Style.NORMAL + (L- len(Text) - 1)*'=' + Style.RESET_ALL)
def Line(L=100, C = 'Blue'):
FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
print(FORE[C] + Style.NORMAL + L*'=' + Style.RESET_ALL)
def Search_List(Key, List): return [s for s in List if Key in s]
def TorchSets(Set):
# Inut: Arrays
# GPU Cuda
if isinstance(Set, (pd.DataFrame, pd.Series)):
Set = Set.values
if torch.cuda.is_available():
if Set.ndim==1:
Out = Variable(torch.from_numpy(Set).type(torch.LongTensor).cuda())
else:
Out = Variable(torch.from_numpy(Set).cuda())
# CPU
else:
if Set.ndim==1:
Out = Variable(torch.from_numpy(Set).type(torch.LongTensor))
else:
Out = Variable(torch.from_numpy(Set))
return Out
# Tensors
X_train_tensor = TorchSets(X_train)
y_train_tensor = TorchSets(y_train)
X_test_tensor = TorchSets(X_test)
y_test_tensor = TorchSets(y_test)
Batch_size = 100
iteration_number = int(4e4)
epochs_number = int(iteration_number / (len(X_train) / Batch_size))
# Pytorch train and test sets
Train_set = torch.utils.data.TensorDataset(X_train_tensor, y_train_tensor)
Test_set = torch.utils.data.TensorDataset(X_test_tensor, y_test_tensor)
# data loader
train_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)
test_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)
# Create MLP_Model
class MLP_Model(nn.Module):
def __init__(self, input_Size, hidden_Size, output_Size):
super(MLP_Model, self).__init__()
# Linear function 1:
self.fc1 = nn.Linear(input_Size, hidden_Size)
# Non-linearity 1
self.relu1 = nn.ReLU()
# Linear function 2:
self.fc2 = nn.Linear(hidden_Size, int(hidden_Size/4))
# Non-linearity 2
self.tanh2 = nn.Tanh()
# Linear function 3:
self.fc3 = nn.Linear(int(hidden_Size/4), output_Size)
def forward(self, x):
# Linear function 1
out = self.fc1(x)
# Non-linearity 1
out = self.relu1(out)
# Linear function 2
out = self.fc2(out)
# Non-linearity 2
out = self.tanh2(out)
# Linear function 4 (readout)
out = self.fc3(out)
return out
def Plot_history(history, Table_Rows = 25, yLim = 2):
fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=[0.6, 0.4],
specs=[[{"type": "scatter"},{"type": "table"}]])
# Left
fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Loss'].astype(float).values.round(4),
line=dict(color='OrangeRed', width= 1.5), name = 'Loss'), 1, 1)
fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Accuracy'].astype(float).values,
line=dict(color='MidnightBlue', width= 1.5), name = 'Accuracy'), 1, 1)
fig.update_layout(legend=dict(x=0, y=1.1, traceorder='reversed', font_size=12),
dragmode='select', plot_bgcolor= 'white', height=600, hovermode='closest',
legend_orientation='h')
fig.update_xaxes(range=[history.Iteration.min(), history.Iteration.max()],
showgrid=True, gridwidth=1, gridcolor='Lightgray',
showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
fig.update_yaxes(range=[0, yLim], showgrid=True, gridwidth=1, gridcolor='Lightgray',
showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
# Right
ind = np.linspace(0, history.shape[0], Table_Rows, endpoint = False).round(0).astype(int)
ind = np.append(ind, history.index[-1])
history = history[history.index.isin(ind)]
T = history.copy()
T[['Loss','Accuracy']] = T[['Loss','Accuracy']].applymap(lambda x: '%.4e' % x)
Temp = []
for i in T.columns:
Temp.append(T.loc[:,i].values)
fig.add_trace(go.Table(header=dict(values = list(history.columns), line_color='darkslategray',
fill_color='Navy', align=['center','center'],
font=dict(color='white', size=12), height=25), columnwidth = [0.4, 0.4, 0.4],
cells=dict(values=Temp, line_color='darkslategray',
fill=dict(color=['Lavender', 'white', 'white']),
align=['center', 'center'], font_size=12,height=20)), 1, 2)
fig.show()
Fitting the model
n_Inst = X_train.shape[1]
n_Att = X_train.shape[1]
input_Size, output_Size = n_Inst, len(Labels)
hidden_Size = 256
# model
model = MLP_Model(input_Size, hidden_Size, output_Size)
# GPU
if torch.cuda.is_available():
model.cuda()
# Cross Entropy Loss
CEL= nn.CrossEntropyLoss()
# Optimizer
learning_rate = 1e-2
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
# Traning the Model
Count = 0
Loss_list = []
Iteration_list = []
Accuracy_list = []
MSE_list = []
MAE_list = []
Steps = 10
Progress_Bar = progressbar.ProgressBar(maxval= int(iteration_number+iteration_number/10),
widgets=[progressbar.Bar('=', '|', '|'), progressbar.Percentage()])
# print('---------------------------------------------------------')
for epoch in range(epochs_number):
for i, (Xtr, ytr) in enumerate(train_loader):
# Variables
Xtr = Variable(Xtr.view(-1, n_Att))
ytr = Variable(ytr)
# Set all gradients to zero
optimizer.zero_grad()
# Forward
Out = model(Xtr.float())
# loss
loss = CEL(Out, ytr.long())
# Backward (Calculating the gradients)
loss.backward()
# Update parameters
optimizer.step()
Count += 1
del Xtr, ytr
# Predictions
if Count % Steps == 0:
# Calculate Accuracy
Correct, Total = 0, 0
# Predictions
for Xts, yts in test_loader:
Xts = Variable(Xts.view(-1, n_Att))
# Forward
Out = model(Xts.float())
# The maximum value of Out
Predicted = torch.max(Out.data, 1)[1]
# Total number of yts
Total += len(yts)
# Total Correct predictions
Correct += (Predicted == yts).sum()
del Xts, yts
# storing loss and iteration
Loss_list.append(loss.data)
Iteration_list.append(Count)
Accuracy_list.append(Correct / float(Total))
Progress_Bar.update(Count)
Progress_Bar.finish()
history = pd.DataFrame({'Iteration': np.array(Iteration_list),
'Loss': np.array([x.cpu().data.numpy() for x in Loss_list]),
'Accuracy': np.array([x.cpu().data.numpy() for x in Accuracy_list])})
del Loss_list, Iteration_list, Accuracy_list
|=========================================================================|100%
Model Performance
Plot_history(history, Table_Rows = 18, yLim = 3)
The confusion matrix allows for visualization of the performance of an algorithm. Note that due to the size of data, here we don't provide a Cross-validation evaluation. In general, this type of evaluation is preferred.
def Confusion_Mat(CM_Train, CM_Test, PD, n_splits = 10):
if n_splits == None:
Titles = ['Train Set', 'Test Set']
else:
Titles = ['Train Set (CV = % i)' % n_splits, 'Test Set (CV = % i)' % n_splits]
CM = [CM_Train, CM_Test]
Cmap = ['Greens', 'YlGn','Blues', 'PuBu']
for i in range(2):
fig, ax = plt.subplots(1, 2, figsize= PD['FS'])
fig.suptitle(Titles[i], weight = 'bold', fontsize = 16)
_ = sns.heatmap(CM[i], annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i], ax = ax[0],
linewidths = 0.2, cbar_kws={"shrink": PD['shrink']})
_ = ax[0].set_title('Confusion Matrix');
Temp = np.round(CM[i].astype('float') / CM[i].sum(axis=1)[:, np.newaxis], 2)
_ = sns.heatmap(Temp,
annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i+1], ax = ax[1],
linewidths = 0.4, vmin=0, vmax=1, cbar_kws={"shrink": PD['shrink']})
_ = ax[1].set_title('Normalized Confusion Matrix');
for a in ax:
_ = a.set_xlabel('Predicted labels')
_ = a.set_ylabel('True labels');
_ = a.xaxis.set_ticklabels(Labels)
_ = a.yaxis.set_ticklabels(Labels)
_ = a.set_aspect(1)
# Train
y_pred = model(X_train_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Train = pd.DataFrame(metrics.classification_report(y_train, y_pred, target_names=Labels, output_dict=True)).T
CM_Train = metrics.confusion_matrix(y_train, y_pred)
# Test
y_pred = model(X_test_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Test = pd.DataFrame(metrics.classification_report(y_test, y_pred, target_names=Labels, output_dict=True)).T
CM_Test = metrics.confusion_matrix(y_test, y_pred)
Reports_Train = Reports_Train.reset_index().rename(columns ={'index': 'Train Set'})
Reports_Test = Reports_Test.reset_index().rename(columns ={'index': 'Test Set'})
display(Reports_Train.style.hide_index().set_properties(**{'background-color': 'HoneyDew', 'color': 'Black'}).\
set_properties(subset=['Train Set'], **{'background-color': 'SeaGreen', 'color': 'White'}))
display(Reports_Test.style.hide_index().set_properties(**{'background-color': 'Azure', 'color': 'Black'}).\
set_properties(subset=['Test Set'], **{'background-color': 'RoyalBlue', 'color': 'White'}))
PD = dict(FS = (20, 8), annot_kws = 10, shrink = .6)
Confusion_Mat(CM_Train, CM_Test, PD = PD, n_splits = None)
| Train Set | precision | recall | f1-score | support |
|---|---|---|---|---|
| 0 | 1.000000 | 1.000000 | 1.000000 | 143.000000 |
| 1 | 1.000000 | 1.000000 | 1.000000 | 143.000000 |
| 2 | 1.000000 | 1.000000 | 1.000000 | 144.000000 |
| 3 | 1.000000 | 1.000000 | 1.000000 | 145.000000 |
| 4 | 1.000000 | 1.000000 | 1.000000 | 144.000000 |
| 5 | 1.000000 | 1.000000 | 1.000000 | 146.000000 |
| 6 | 1.000000 | 1.000000 | 1.000000 | 145.000000 |
| 7 | 1.000000 | 1.000000 | 1.000000 | 144.000000 |
| 8 | 1.000000 | 1.000000 | 1.000000 | 144.000000 |
| 9 | 1.000000 | 1.000000 | 1.000000 | 145.000000 |
| accuracy | 1.000000 | 1.000000 | 1.000000 | 1.000000 |
| macro avg | 1.000000 | 1.000000 | 1.000000 | 1443.000000 |
| weighted avg | 1.000000 | 1.000000 | 1.000000 | 1443.000000 |
| Test Set | precision | recall | f1-score | support |
|---|---|---|---|---|
| 0 | 0.841270 | 0.868852 | 0.854839 | 61.000000 |
| 1 | 0.931034 | 0.870968 | 0.900000 | 62.000000 |
| 2 | 0.750000 | 0.774194 | 0.761905 | 62.000000 |
| 3 | 0.739726 | 0.870968 | 0.800000 | 62.000000 |
| 4 | 0.952381 | 0.967742 | 0.960000 | 62.000000 |
| 5 | 0.870370 | 0.758065 | 0.810345 | 62.000000 |
| 6 | 0.704918 | 0.693548 | 0.699187 | 62.000000 |
| 7 | 0.912281 | 0.838710 | 0.873950 | 62.000000 |
| 8 | 0.709677 | 0.709677 | 0.709677 | 62.000000 |
| 9 | 0.875000 | 0.903226 | 0.888889 | 62.000000 |
| accuracy | 0.825525 | 0.825525 | 0.825525 | 0.825525 |
| macro avg | 0.828666 | 0.825595 | 0.825879 | 619.000000 |
| weighted avg | 0.828645 | 0.825525 | 0.825832 | 619.000000 |